{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import collections\n",
    "import gym\n",
    "import random\n",
    "import tensorflow.contrib.layers as layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ENV = \"CartPole-v0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MEMORY_SIZE = 10000\n",
    "EPISODES = 500\n",
    "MAX_STEP = 500\n",
    "BATCH_SIZE = 32\n",
    "UPDATE_PERIOD = 200  # update target network parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##built class for the DQN\n",
    "class DeepQNetwork():\n",
    "    def __init__(self , env , sess=None , gamma = 0.8, epsilon = 0.8 ):\n",
    "        self.gamma = gamma\n",
    "        self.epsilon = epsilon\n",
    "        self.action_dim = env.action_space.n\n",
    "        self.state_dim = env.observation_space.shape[0]\n",
    "        self.network()\n",
    "        self.sess = sess\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "        tf.summary.FileWriter(\"DQN/summaries\" , sess.graph )\n",
    "        \n",
    "    # net_frame using for creating Q & target network\n",
    "    def net_frame(self , hiddens, inpt, num_actions, scope, reuse=None):\n",
    "        with tf.variable_scope(scope, reuse=reuse):\n",
    "            out = inpt  \n",
    "            for hidden in hiddens:\n",
    "                out = layers.fully_connected(out,  num_outputs=hidden, activation_fn=tf.nn.relu)\n",
    "            out = layers.fully_connected(out, num_outputs=num_actions, activation_fn=None) \n",
    "            return out\n",
    "        \n",
    "    # create q_network & target_network     \n",
    "    def network(self):       \n",
    "        # q_network\n",
    "        self.inputs_q = tf.placeholder(dtype = tf.float32 , shape = [None , self.state_dim] , name = \"inputs_q\")\n",
    "        scope_var = \"q_network\"    \n",
    "        self.q_value = self.net_frame([64] , self.inputs_q , self.action_dim , scope_var , reuse = True )\n",
    "            \n",
    "        # target_network\n",
    "        self.inputs_target = tf.placeholder(dtype = tf.float32 , shape = [None , self.state_dim] , name = \"inputs_target\")\n",
    "        scope_tar = \"target_network\"    \n",
    "        self.q_target = self.net_frame([64] , self.inputs_target , self.action_dim , scope_tar )\n",
    "               \n",
    "        with tf.variable_scope(\"loss\"):\n",
    "#            #【方案一】\n",
    "#             self.target = tf.placeholder(dtype = tf.float32 , shape = [None , self.action_dim] , name = \"target\")\n",
    "#             self.loss = tf.reduce_mean( tf.square(self.q_value - self.target))\n",
    "            #【方案二】\n",
    "            self.action = tf.placeholder(dtype = tf.int32 , shape = [ None ] , name = \"action\")\n",
    "            action_one_hot = tf.one_hot(self.action , self.action_dim )\n",
    "            q_action = tf.reduce_sum( tf.multiply(self.q_value , action_one_hot) , axis = 1 ) \n",
    "            \n",
    "            self.target =  tf.placeholder(dtype = tf.float32 , shape =  [None ] , name = \"target\")\n",
    "            self.loss = tf.reduce_mean( tf.square(q_action - self.target))\n",
    "\n",
    "        with tf.variable_scope(\"train\"):\n",
    "            optimizer = tf.train.RMSPropOptimizer(0.001)\n",
    "            self.train_op = optimizer.minimize(self.loss)    \n",
    "    \n",
    "    # training\n",
    "    def train(self , state , reward , action , state_next , done):\n",
    "        q , q_target = self.sess.run([self.q_value , self.q_target] , \n",
    "                                     feed_dict={self.inputs_q : state , self.inputs_target : state_next } )\n",
    "#         #【方案一】\n",
    "#         target = reward + self.gamma * np.max(q_target , axis = 1)*(1.0 - done)\n",
    "        \n",
    "#         self.reform_target = q.copy()\n",
    "#         batch_index = np.arange(BATCH_SIZE , dtype = np.int32)\n",
    "#         self.reform_target[batch_index , action] = target\n",
    "            \n",
    "#         loss , _ = self.sess.run([self.loss , self.train_op] , feed_dict={self.inputs_q: state , self.target: self.reform_target} )\n",
    "\n",
    "        #【方案二】\n",
    "        q_target_best = np.max(q_target , axis = 1)\n",
    "        q_target_best_mask = ( 1.0 - done) * q_target_best\n",
    "        \n",
    "        target = reward + self.gamma * q_target_best_mask\n",
    "        \n",
    "        loss , _ = self.sess.run([self.loss , self.train_op] , \n",
    "                                 feed_dict={self.inputs_q: state , self.target:target , self.action:action} )    \n",
    "    # chose action\n",
    "    def chose_action(self , current_state):\n",
    "        current_state = current_state[np.newaxis , :]  #*** array dim: (xx,)  --> (1 , xx) ***\n",
    "        q = self.sess.run(self.q_value , feed_dict={self.inputs_q : current_state} )\n",
    "        \n",
    "        # e-greedy\n",
    "        if np.random.random() < self.epsilon:\n",
    "            action_chosen = np.random.randint(0 , self.action_dim)\n",
    "        else:\n",
    "            action_chosen = np.argmax(q)\n",
    "        \n",
    "        return action_chosen\n",
    "         \n",
    "    #upadate parmerters\n",
    "    def update_prmt(self):\n",
    "        q_prmts = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES ,   \"q_network\"  )\n",
    "        target_prmts = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES,  \"target_network\"  )\n",
    "        self.sess.run( [tf.assign(t , q)for t,q in zip(target_prmts , q_prmts)])  #***\n",
    "        print(\"updating target-network parmeters...\")\n",
    "        \n",
    "    def decay_epsilon(self):\n",
    "        if self.epsilon > 0.03:\n",
    "            self.epsilon = self.epsilon - 0.02"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# memory for momery replay\n",
    "memory = []\n",
    "Transition = collections.namedtuple(\"Transition\" , [\"state\", \"action\" , \"reward\" , \"next_state\" , \"done\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-06-16 10:32:02,009] Making new env: CartPole-v0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode = 0 ] step = 125\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode = 1 ] step = 24\n",
      "[episode = 2 ] step = 45\n",
      "[episode = 3 ] step = 51\n",
      "updating target-network parmeters...\n",
      "[episode = 4 ] step = 78\n",
      "[episode = 5 ] step = 58\n",
      "[episode = 6 ] step = 32\n",
      "[episode = 7 ] step = 28\n",
      "[episode = 8 ] step = 34\n",
      "[episode = 9 ] step = 11\n",
      "[episode = 10 ] step = 9\n",
      "[episode = 11 ] step = 8\n",
      "[episode = 12 ] step = 8\n",
      "updating target-network parmeters...\n",
      "[episode = 13 ] step = 8\n",
      "[episode = 14 ] step = 30\n",
      "[episode = 15 ] step = 45\n",
      "[episode = 16 ] step = 17\n",
      "[episode = 17 ] step = 8\n",
      "[episode = 18 ] step = 13\n",
      "[episode = 19 ] step = 9\n",
      "[episode = 20 ] step = 9\n",
      "[episode = 21 ] step = 10\n",
      "[episode = 22 ] step = 35\n",
      "updating target-network parmeters...\n",
      "[episode = 23 ] step = 11\n",
      "[episode = 24 ] step = 33\n",
      "[episode = 25 ] step = 39\n",
      "[episode = 26 ] step = 30\n",
      "[episode = 27 ] step = 22\n",
      "[episode = 28 ] step = 30\n",
      "[episode = 29 ] step = 19\n",
      "[episode = 30 ] step = 16\n",
      "updating target-network parmeters...\n",
      "[episode = 31 ] step = 38\n",
      "[episode = 32 ] step = 34\n",
      "[episode = 33 ] step = 49\n",
      "[episode = 34 ] step = 34\n",
      "[episode = 35 ] step = 18\n",
      "[episode = 36 ] step = 10\n",
      "updating target-network parmeters...\n",
      "[episode = 37 ] step = 20\n",
      "[episode = 38 ] step = 33\n",
      "[episode = 39 ] step = 62\n",
      "[episode = 40 ] step = 44\n",
      "[episode = 41 ] step = 26\n",
      "updating target-network parmeters...\n",
      "[episode = 42 ] step = 32\n",
      "[episode = 43 ] step = 10\n",
      "[episode = 44 ] step = 14\n",
      "[episode = 45 ] step = 45\n",
      "[episode = 46 ] step = 9\n",
      "[episode = 47 ] step = 9\n",
      "[episode = 48 ] step = 8\n",
      "[episode = 49 ] step = 9\n",
      "[episode = 50 ] step = 13\n",
      "[episode = 51 ] step = 9\n",
      "[episode = 52 ] step = 9\n",
      "[episode = 53 ] step = 8\n",
      "[episode = 54 ] step = 9\n",
      "[episode = 55 ] step = 9\n",
      "[episode = 56 ] step = 8\n",
      "updating target-network parmeters...\n",
      "[episode = 57 ] step = 11\n",
      "[episode = 58 ] step = 21\n",
      "[episode = 59 ] step = 35\n",
      "[episode = 60 ] step = 19\n",
      "[episode = 61 ] step = 11\n",
      "[episode = 62 ] step = 9\n",
      "[episode = 63 ] step = 22\n",
      "[episode = 64 ] step = 17\n",
      "[episode = 65 ] step = 9\n",
      "[episode = 66 ] step = 9\n",
      "[episode = 67 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode = 68 ] step = 32\n",
      "[episode = 69 ] step = 14\n",
      "[episode = 70 ] step = 9\n",
      "[episode = 71 ] step = 11\n",
      "[episode = 72 ] step = 9\n",
      "[episode = 73 ] step = 17\n",
      "[episode = 74 ] step = 14\n",
      "[episode = 75 ] step = 22\n",
      "[episode = 76 ] step = 30\n",
      "[episode = 77 ] step = 8\n",
      "[episode = 78 ] step = 13\n",
      "[episode = 79 ] step = 10\n",
      "[episode = 80 ] step = 10\n",
      "updating target-network parmeters...\n",
      "[episode = 81 ] step = 9\n",
      "[episode = 82 ] step = 39\n",
      "[episode = 83 ] step = 30\n",
      "[episode = 84 ] step = 9\n",
      "[episode = 85 ] step = 33\n",
      "[episode = 86 ] step = 11\n",
      "[episode = 87 ] step = 9\n",
      "[episode = 88 ] step = 25\n",
      "[episode = 89 ] step = 10\n",
      "[episode = 90 ] step = 15\n",
      "updating target-network parmeters...\n",
      "[episode = 91 ] step = 21\n",
      "[episode = 92 ] step = 86\n",
      "[episode = 93 ] step = 19\n",
      "[episode = 94 ] step = 11\n",
      "[episode = 95 ] step = 9\n",
      "[episode = 96 ] step = 56\n",
      "updating target-network parmeters...\n",
      "[episode = 97 ] step = 9\n",
      "[episode = 98 ] step = 9\n",
      "[episode = 99 ] step = 10\n",
      "[episode = 100 ] step = 24\n",
      "[episode = 101 ] step = 20\n",
      "[episode = 102 ] step = 22\n",
      "[episode = 103 ] step = 36\n",
      "[episode = 104 ] step = 38\n",
      "updating target-network parmeters...\n",
      "[episode = 105 ] step = 34\n",
      "[episode = 106 ] step = 38\n",
      "[episode = 107 ] step = 92\n",
      "[episode = 108 ] step = 31\n",
      "updating target-network parmeters...\n",
      "[episode = 109 ] step = 60\n",
      "[episode = 110 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode = 111 ] step = 68\n",
      "[episode = 112 ] step = 42\n",
      "[episode = 113 ] step = 22\n",
      "[episode = 114 ] step = 31\n",
      "[episode = 115 ] step = 31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating target-network parmeters...\n",
      "[episode = 116 ] step = 39\n",
      "[episode = 117 ] step = 44\n",
      "[episode = 118 ] step = 34\n",
      "[episode = 119 ] step = 29\n",
      "[episode = 120 ] step = 48\n",
      "updating target-network parmeters...\n",
      "[episode = 121 ] step = 44\n",
      "[episode = 122 ] step = 33\n",
      "[episode = 123 ] step = 19\n",
      "[episode = 124 ] step = 44\n",
      "[episode = 125 ] step = 56\n",
      "updating target-network parmeters...\n",
      "[episode = 126 ] step = 43\n",
      "[episode = 127 ] step = 36\n",
      "[episode = 128 ] step = 24\n",
      "[episode = 129 ] step = 33\n",
      "[episode = 130 ] step = 33\n",
      "updating target-network parmeters...\n",
      "[episode = 131 ] step = 52\n",
      "[episode = 132 ] step = 56\n",
      "[episode = 133 ] step = 59\n",
      "[episode = 134 ] step = 39\n",
      "updating target-network parmeters...\n",
      "[episode = 135 ] step = 48\n",
      "[episode = 136 ] step = 44\n",
      "[episode = 137 ] step = 39\n",
      "[episode = 138 ] step = 33\n",
      "updating target-network parmeters...\n",
      "[episode = 139 ] step = 135\n",
      "[episode = 140 ] step = 36\n",
      "[episode = 141 ] step = 38\n",
      "[episode = 142 ] step = 50\n",
      "updating target-network parmeters...\n",
      "[episode = 143 ] step = 44\n",
      "[episode = 144 ] step = 61\n",
      "[episode = 145 ] step = 48\n",
      "updating target-network parmeters...\n",
      "[episode = 146 ] step = 76\n",
      "[episode = 147 ] step = 106\n",
      "[episode = 148 ] step = 45\n",
      "updating target-network parmeters...\n",
      "[episode = 149 ] step = 74\n",
      "[episode = 150 ] step = 33\n",
      "[episode = 151 ] step = 61\n",
      "[episode = 152 ] step = 40\n",
      "updating target-network parmeters...\n",
      "[episode = 153 ] step = 62\n",
      "[episode = 154 ] step = 69\n",
      "[episode = 155 ] step = 42\n",
      "updating target-network parmeters...\n",
      "[episode = 156 ] step = 50\n",
      "[episode = 157 ] step = 58\n",
      "[episode = 158 ] step = 42\n",
      "[episode = 159 ] step = 65\n",
      "updating target-network parmeters...\n",
      "[episode = 160 ] step = 55\n",
      "[episode = 161 ] step = 41\n",
      "[episode = 162 ] step = 63\n",
      "updating target-network parmeters...\n",
      "[episode = 163 ] step = 57\n",
      "[episode = 164 ] step = 68\n",
      "[episode = 165 ] step = 48\n",
      "[episode = 166 ] step = 51\n",
      "updating target-network parmeters...\n",
      "[episode = 167 ] step = 80\n",
      "[episode = 168 ] step = 56\n",
      "[episode = 169 ] step = 49\n",
      "updating target-network parmeters...\n",
      "[episode = 170 ] step = 55\n",
      "[episode = 171 ] step = 73\n",
      "[episode = 172 ] step = 38\n",
      "[episode = 173 ] step = 56\n",
      "updating target-network parmeters...\n",
      "[episode = 174 ] step = 58\n",
      "[episode = 175 ] step = 94\n",
      "updating target-network parmeters...\n",
      "[episode = 176 ] step = 52\n",
      "[episode = 177 ] step = 46\n",
      "[episode = 178 ] step = 53\n",
      "[episode = 179 ] step = 67\n",
      "updating target-network parmeters...\n",
      "[episode = 180 ] step = 41\n",
      "[episode = 181 ] step = 79\n",
      "updating target-network parmeters...\n",
      "[episode = 182 ] step = 151\n",
      "[episode = 183 ] step = 51\n",
      "[episode = 184 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode = 185 ] step = 48\n",
      "[episode = 186 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode = 187 ] step = 69\n",
      "[episode = 188 ] step = 60\n",
      "updating target-network parmeters...\n",
      "[episode = 189 ] step = 172\n",
      "[episode = 190 ] step = 53\n",
      "[episode = 191 ] step = 56\n",
      "updating target-network parmeters...\n",
      "[episode = 192 ] step = 54\n",
      "[episode = 193 ] step = 93\n",
      "[episode = 194 ] step = 48\n",
      "updating target-network parmeters...\n",
      "[episode = 195 ] step = 89\n",
      "updating target-network parmeters...\n",
      "[episode = 196 ] step = 163\n",
      "[episode = 197 ] step = 106\n",
      "[episode = 198 ] step = 61\n",
      "updating target-network parmeters...\n",
      "[episode = 199 ] step = 61\n",
      "[episode = 200 ] step = 54\n",
      "[episode = 201 ] step = 67\n",
      "updating target-network parmeters...\n",
      "[episode = 202 ] step = 53\n",
      "[episode = 203 ] step = 80\n",
      "[episode = 204 ] step = 64\n",
      "updating target-network parmeters...\n",
      "[episode = 205 ] step = 68\n",
      "[episode = 206 ] step = 76\n",
      "[episode = 207 ] step = 66\n",
      "updating target-network parmeters...\n",
      "[episode = 208 ] step = 78\n",
      "[episode = 209 ] step = 59\n",
      "[episode = 210 ] step = 70\n",
      "updating target-network parmeters...\n",
      "[episode = 211 ] step = 68\n",
      "[episode = 212 ] step = 67\n",
      "updating target-network parmeters...\n",
      "[episode = 213 ] step = 73\n",
      "[episode = 214 ] step = 84\n",
      "[episode = 215 ] step = 62\n",
      "updating target-network parmeters...\n",
      "[episode = 216 ] step = 56\n",
      "[episode = 217 ] step = 98\n",
      "[episode = 218 ] step = 59\n",
      "updating target-network parmeters...\n",
      "[episode = 219 ] step = 66\n",
      "[episode = 220 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode = 221 ] step = 184\n",
      "updating target-network parmeters...\n",
      "[episode = 222 ] step = 92\n",
      "[episode = 223 ] step = 96\n",
      "updating target-network parmeters...\n",
      "[episode = 224 ] step = 78\n",
      "[episode = 225 ] step = 165\n",
      "updating target-network parmeters...\n",
      "[episode = 226 ] step = 77\n",
      "[episode = 227 ] step = 63\n",
      "updating target-network parmeters...\n",
      "[episode = 228 ] step = 108\n",
      "[episode = 229 ] step = 63\n",
      "[episode = 230 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode = 231 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode = 232 ] step = 83\n",
      "[episode = 233 ] step = 88\n",
      "[episode = 234 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode = 235 ] step = 106\n",
      "[episode = 236 ] step = 97\n",
      "updating target-network parmeters...\n",
      "[episode = 237 ] step = 95\n",
      "[episode = 238 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode = 239 ] step = 107\n",
      "[episode = 240 ] step = 93\n",
      "updating target-network parmeters...\n",
      "[episode = 241 ] step = 96\n",
      "[episode = 242 ] step = 93\n",
      "updating target-network parmeters...\n",
      "[episode = 243 ] step = 72\n",
      "[episode = 244 ] step = 67\n",
      "updating target-network parmeters...\n",
      "[episode = 245 ] step = 89\n",
      "[episode = 246 ] step = 10\n",
      "[episode = 247 ] step = 12\n",
      "[episode = 248 ] step = 67\n",
      "[episode = 249 ] step = 76\n",
      "updating target-network parmeters...\n",
      "[episode = 250 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode = 251 ] step = 107\n",
      "[episode = 252 ] step = 85\n",
      "[episode = 253 ] step = 73\n",
      "updating target-network parmeters...\n",
      "[episode = 254 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode = 255 ] step = 199\n",
      "[episode = 256 ] step = 125\n",
      "updating target-network parmeters...\n",
      "[episode = 257 ] step = 64\n",
      "[episode = 258 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode = 259 ] step = 93\n",
      "[episode = 260 ] step = 84\n",
      "[episode = 261 ] step = 10\n",
      "updating target-network parmeters...\n",
      "[episode = 262 ] step = 72\n",
      "[episode = 263 ] step = 79\n",
      "updating target-network parmeters...\n",
      "[episode = 264 ] step = 80\n",
      "[episode = 265 ] step = 70\n",
      "[episode = 266 ] step = 88\n",
      "updating target-network parmeters...\n",
      "[episode = 267 ] step = 112\n",
      "[episode = 268 ] step = 88\n",
      "updating target-network parmeters...\n",
      "[episode = 269 ] step = 80\n",
      "updating target-network parmeters...\n",
      "[episode = 270 ] step = 151\n",
      "[episode = 271 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode = 272 ] step = 177\n",
      "updating target-network parmeters...\n",
      "[episode = 273 ] step = 179\n",
      "updating target-network parmeters...\n",
      "[episode = 274 ] step = 127\n",
      "[episode = 275 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode = 276 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode = 277 ] step = 138\n",
      "[episode = 278 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode = 279 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode = 280 ] step = 148\n",
      "[episode = 281 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode = 282 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode = 283 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode = 284 ] step = 139\n",
      "[episode = 285 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode = 286 ] step = 106\n",
      "[episode = 287 ] step = 74\n",
      "updating target-network parmeters...\n",
      "[episode = 288 ] step = 163\n",
      "[episode = 289 ] step = 113\n",
      "updating target-network parmeters...\n",
      "[episode = 290 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode = 291 ] step = 117\n",
      "[episode = 292 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode = 293 ] step = 91\n",
      "[episode = 294 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode = 295 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode = 296 ] step = 193\n",
      "updating target-network parmeters...\n",
      "[episode = 297 ] step = 146\n",
      "updating target-network parmeters...\n",
      "[episode = 298 ] step = 183\n",
      "updating target-network parmeters...\n",
      "[episode = 299 ] step = 197\n",
      "[episode = 300 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode = 301 ] step = 113\n",
      "updating target-network parmeters...\n",
      "[episode = 302 ] step = 132\n",
      "[episode = 303 ] step = 174\n",
      "updating target-network parmeters...\n",
      "[episode = 304 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode = 305 ] step = 196\n",
      "updating target-network parmeters...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode = 306 ] step = 176\n",
      "updating target-network parmeters...\n",
      "[episode = 307 ] step = 188\n",
      "updating target-network parmeters...\n",
      "[episode = 308 ] step = 130\n",
      "[episode = 309 ] step = 185\n",
      "updating target-network parmeters...\n",
      "[episode = 310 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 311 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 312 ] step = 196\n",
      "updating target-network parmeters...\n",
      "[episode = 313 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 314 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode = 315 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 316 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode = 317 ] step = 144\n",
      "updating target-network parmeters...\n",
      "[episode = 318 ] step = 166\n",
      "updating target-network parmeters...\n",
      "[episode = 319 ] step = 184\n",
      "updating target-network parmeters...\n",
      "[episode = 320 ] step = 197\n",
      "[episode = 321 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode = 322 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode = 323 ] step = 133\n",
      "updating target-network parmeters...\n",
      "[episode = 324 ] step = 195\n",
      "[episode = 325 ] step = 169\n",
      "updating target-network parmeters...\n",
      "[episode = 326 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 327 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 328 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 329 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 330 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 331 ] step = 158\n",
      "updating target-network parmeters...\n",
      "[episode = 332 ] step = 185\n",
      "updating target-network parmeters...\n",
      "[episode = 333 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode = 334 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode = 335 ] step = 168\n",
      "[episode = 336 ] step = 156\n",
      "updating target-network parmeters...\n",
      "[episode = 337 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode = 338 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 339 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 340 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 341 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 342 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 343 ] step = 62\n",
      "updating target-network parmeters...\n",
      "[episode = 344 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 345 ] step = 187\n",
      "[episode = 346 ] step = 71\n",
      "updating target-network parmeters...\n",
      "[episode = 347 ] step = 153\n",
      "[episode = 348 ] step = 141\n",
      "updating target-network parmeters...\n",
      "[episode = 349 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode = 350 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode = 351 ] step = 193\n",
      "[episode = 352 ] step = 94\n",
      "updating target-network parmeters...\n",
      "[episode = 353 ] step = 125\n",
      "updating target-network parmeters...\n",
      "[episode = 354 ] step = 119\n",
      "[episode = 355 ] step = 60\n",
      "updating target-network parmeters...\n",
      "[episode = 356 ] step = 185\n",
      "[episode = 357 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode = 358 ] step = 49\n",
      "updating target-network parmeters...\n",
      "[episode = 359 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 360 ] step = 199\n",
      "[episode = 361 ] step = 61\n",
      "[episode = 362 ] step = 17\n",
      "[episode = 363 ] step = 23\n",
      "updating target-network parmeters...\n",
      "[episode = 364 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 365 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode = 366 ] step = 180\n",
      "[episode = 367 ] step = 21\n",
      "[episode = 368 ] step = 69\n",
      "updating target-network parmeters...\n",
      "[episode = 369 ] step = 153\n",
      "[episode = 370 ] step = 34\n",
      "[episode = 371 ] step = 34\n",
      "updating target-network parmeters...\n",
      "[episode = 372 ] step = 60\n",
      "[episode = 373 ] step = 111\n",
      "updating target-network parmeters...\n",
      "[episode = 374 ] step = 122\n",
      "[episode = 375 ] step = 23\n",
      "[episode = 376 ] step = 53\n",
      "updating target-network parmeters...\n",
      "[episode = 377 ] step = 107\n",
      "[episode = 378 ] step = 28\n",
      "[episode = 379 ] step = 22\n",
      "[episode = 380 ] step = 74\n",
      "[episode = 381 ] step = 20\n",
      "updating target-network parmeters...\n",
      "[episode = 382 ] step = 39\n",
      "[episode = 383 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode = 384 ] step = 164\n",
      "[episode = 385 ] step = 76\n",
      "updating target-network parmeters...\n",
      "[episode = 386 ] step = 159\n",
      "updating target-network parmeters...\n",
      "[episode = 387 ] step = 134\n",
      "[episode = 388 ] step = 100\n",
      "updating target-network parmeters...\n",
      "[episode = 389 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode = 390 ] step = 113\n",
      "[episode = 391 ] step = 143\n",
      "updating target-network parmeters...\n",
      "[episode = 392 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode = 393 ] step = 130\n",
      "[episode = 394 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode = 395 ] step = 122\n",
      "[episode = 396 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode = 397 ] step = 114\n",
      "[episode = 398 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode = 399 ] step = 104\n",
      "updating target-network parmeters...\n",
      "[episode = 400 ] step = 109\n",
      "[episode = 401 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode = 402 ] step = 121\n",
      "[episode = 403 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode = 404 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode = 405 ] step = 166\n",
      "[episode = 406 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode = 407 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode = 408 ] step = 125\n",
      "[episode = 409 ] step = 110\n",
      "[episode = 410 ] step = 61\n",
      "updating target-network parmeters...\n",
      "[episode = 411 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode = 412 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 413 ] step = 199\n",
      "[episode = 414 ] step = 94\n",
      "updating target-network parmeters...\n",
      "[episode = 415 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode = 416 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 417 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 418 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 419 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 420 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 421 ] step = 163\n",
      "[episode = 422 ] step = 61\n",
      "updating target-network parmeters...\n",
      "[episode = 423 ] step = 154\n",
      "updating target-network parmeters...\n",
      "[episode = 424 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode = 425 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 426 ] step = 181\n",
      "updating target-network parmeters...\n",
      "[episode = 427 ] step = 150\n",
      "[episode = 428 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode = 429 ] step = 134\n",
      "[episode = 430 ] step = 87\n",
      "updating target-network parmeters...\n",
      "[episode = 431 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode = 432 ] step = 199\n",
      "[episode = 433 ] step = 73\n",
      "updating target-network parmeters...\n",
      "[episode = 434 ] step = 116\n",
      "[episode = 435 ] step = 79\n",
      "updating target-network parmeters...\n",
      "[episode = 436 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 437 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 438 ] step = 160\n",
      "updating target-network parmeters...\n",
      "[episode = 439 ] step = 143\n",
      "updating target-network parmeters...\n",
      "[episode = 440 ] step = 197\n",
      "[episode = 441 ] step = 35\n",
      "updating target-network parmeters...\n",
      "[episode = 442 ] step = 183\n",
      "updating target-network parmeters...\n",
      "[episode = 443 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 444 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 445 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 446 ] step = 154\n",
      "updating target-network parmeters...\n",
      "[episode = 447 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 448 ] step = 195\n",
      "updating target-network parmeters...\n",
      "[episode = 449 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 450 ] step = 186\n",
      "[episode = 451 ] step = 188\n",
      "updating target-network parmeters...\n",
      "[episode = 452 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode = 453 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 454 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 455 ] step = 186\n",
      "updating target-network parmeters...\n",
      "[episode = 456 ] step = 169\n",
      "updating target-network parmeters...\n",
      "[episode = 457 ] step = 190\n",
      "[episode = 458 ] step = 25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating target-network parmeters...\n",
      "[episode = 459 ] step = 90\n",
      "[episode = 460 ] step = 52\n",
      "[episode = 461 ] step = 36\n",
      "updating target-network parmeters...\n",
      "[episode = 462 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode = 463 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode = 464 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 465 ] step = 199\n",
      "[episode = 466 ] step = 79\n",
      "updating target-network parmeters...\n",
      "[episode = 467 ] step = 199\n",
      "[episode = 468 ] step = 46\n",
      "[episode = 469 ] step = 15\n",
      "updating target-network parmeters...\n",
      "[episode = 470 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 471 ] step = 50\n",
      "[episode = 472 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode = 473 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 474 ] step = 188\n",
      "updating target-network parmeters...\n",
      "[episode = 475 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 476 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 477 ] step = 71\n",
      "[episode = 478 ] step = 89\n",
      "[episode = 479 ] step = 66\n",
      "updating target-network parmeters...\n",
      "[episode = 480 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode = 481 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode = 482 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 483 ] step = 198\n",
      "updating target-network parmeters...\n",
      "[episode = 484 ] step = 183\n",
      "updating target-network parmeters...\n",
      "[episode = 485 ] step = 146\n",
      "[episode = 486 ] step = 146\n",
      "updating target-network parmeters...\n",
      "[episode = 487 ] step = 199\n",
      "[episode = 488 ] step = 36\n",
      "updating target-network parmeters...\n",
      "[episode = 489 ] step = 68\n",
      "updating target-network parmeters...\n",
      "[episode = 490 ] step = 199\n",
      "[episode = 491 ] step = 84\n",
      "updating target-network parmeters...\n",
      "[episode = 492 ] step = 93\n",
      "[episode = 493 ] step = 126\n",
      "updating target-network parmeters...\n",
      "[episode = 494 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode = 495 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 496 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode = 497 ] step = 165\n",
      "[episode = 498 ] step = 125\n",
      "updating target-network parmeters...\n",
      "[episode = 499 ] step = 170\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    env = gym.make(ENV)\n",
    "    with tf.Session() as sess:\n",
    "        DQN = DeepQNetwork(env , sess )\n",
    "        update_iter = 0\n",
    "        step_his = []\n",
    "        for episode in range(EPISODES):\n",
    "            state = env.reset()\n",
    "            env.render() \n",
    "            reward_all = 0\n",
    "            #training\n",
    "            for step in range(MAX_STEP):\n",
    "                action = DQN.chose_action(state)\n",
    "                next_state , reward , done , _ = env.step(action)\n",
    "                reward_all += reward \n",
    "\n",
    "                if len(memory) > MEMORY_SIZE:\n",
    "                    memory.pop(0)\n",
    "                memory.append(Transition(state, action , reward , next_state , float(done)))\n",
    "\n",
    "                if len(memory) > BATCH_SIZE * 4:\n",
    "                    batch_transition = random.sample(memory , BATCH_SIZE)\n",
    "                    #***\n",
    "                    batch_state, batch_action, batch_reward, batch_next_state, batch_done = map(np.array , zip(*batch_transition))  \n",
    "                    DQN.train(state = batch_state ,\n",
    "                              reward = batch_reward , \n",
    "                              action = batch_action , \n",
    "                              state_next = batch_next_state,\n",
    "                              done = batch_done\n",
    "                             )\n",
    "                    update_iter += 1\n",
    "\n",
    "                if update_iter % UPDATE_PERIOD == 0:\n",
    "                    DQN.update_prmt()\n",
    "                \n",
    "                if update_iter % 200 == 0:\n",
    "                    DQN.decay_epsilon()\n",
    "\n",
    "                if done:\n",
    "                    print(\"[episode = {} ] step = {}\".format(episode , step))\n",
    "                    break\n",
    "                    \n",
    "                state = next_state\n",
    "            "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
